# -*- coding: utf-8 -*-
'''
Created on 11-11-2012

@author: LONG HOANG GIANG
'''


from urlparse import urljoin
import datetime
import os
import sys
import re
import workerpool
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import commonlib, html2text
from Model.storymodelv2 import StoryModel


useragent = "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16"

class AppTruyen():
    
    urlformat = "http://m.apptruyen.com/doc-truyen/a/{0}/1.html"
    
    def __init__(self, id):
        self.url = self.urlformat.format(id)
        
    def getDetail(self, url):
        tree = commonlib.loadweb(url).build_tree()
        ndNode = tree.xpath("//div[@class='noidung']")
        commonlib.Etree.clean_elements(ndNode, "span", {"class": "copyright"})
        detail = html2text.html2text(commonlib.Etree.tostring(ndNode)).strip()
        detail = re.sub(ur"\n", "<br />", detail)
        return detail
        

    def getChapterList(self):
        print '=====>  GET CHAPTER LIST   <====='
        url = self.url
        data = []
        ic = 0
        while True and ic < 10000:
            tree = commonlib.loadweb(url, user_agent=useragent).build_tree()
            for item in tree.xpath("//div[@class='cacchuong']/h3[contains(., 'Các chương của truyện')]/following-sibling::*[1]/li/a".decode('utf-8')):
                title = commonlib.stringify(item)
                title = re.sub(r"--\d+--", "", title)
                link = item.get('href', '')
                if link == '': continue
                link = urljoin(url, link)
                data.append({'title': title, 'url': link})
            ic += 1
            nextNode = tree.xpath("//div[@class='page-navigation']/a/button[@class='next']/..")
            if len(nextNode) == 0: break
            href = nextNode[0].get('href', '')
            if href.strip() == '': break
            url = urljoin(url, href)
        return data
    
    def preloadNCache(self, data):
        print '==================== PRELOAD AND CACHE ====================='
        def loadweb(item):
            commonlib.loadweb(item['url'])
        pool = workerpool.WorkerPool(size=5)
        pool.map(loadweb, data)
        pool.shutdown()
        pool.wait()
    
    def process(self):
        
        data = self.getChapterList()
        self.preloadNCache(data)
        smodel = StoryModel(database)
        smodel.open(True)
        
        for item in data:
            print '---------> process: {0}'.format(item['title'])
            detail = self.getDetail(item['url'])
            if detail == '' or item['title'].strip() == '': continue
            smodel.add_story(commonlib.toUpper(item['title']), detail, 0)
        smodel.close()
        
        
        

if __name__ == '__main__':
    
    database = "/longhoanggiang/database/{0}".format("tgtuchan")
    apptruyen = AppTruyen("429")
    apptruyen.process()
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)